@article{DBLP:journals/corr/LiuL17d,
  author    = {Yang Liu and
                 Mirella Lapata},
                   title     = {Learning Structured Text Representations},
                     journal   = {CoRR},
                       volume    = {abs/1705.09207},
                         year      = {2017},
                           url       = {http://arxiv.org/abs/1705.09207},
                             archivePrefix = {arXiv},
                               eprint    = {1705.09207},
                                 timestamp = {Wed, 07 Jun 2017 14:41:46 +0200},
                                   biburl    = {http://dblp.org/rec/bib/journals/corr/LiuL17d},
                                     bibsource = {dblp computer science bibliography, http://dblp.org}
                                     }

@article{sennrich2016linguistic,
  title={Linguistic Input Features Improve Neural Machine Translation},
    author={Sennrich, Rico and Haddow, Barry},
      journal={arXiv preprint arXiv:1606.02892},
        year={2016}
        }

@inproceedings{Li2016
  author    = {Yujia Li and
               Daniel Tarlow and
               Marc Brockschmidt and
               Richard S. Zemel},
  title     = {Gated Graph Sequence Neural Networks},
  booktitle = {4th International Conference on Learning Representations, {ICLR} 2016,
               San Juan, Puerto Rico, May 2-4, 2016, Conference Track Proceedings},
  year      = {2016},
  crossref  = {DBLP:conf/iclr/2016},
  url       = {http://arxiv.org/abs/1511.05493},
  timestamp = {Thu, 25 Jul 2019 14:25:40 +0200},
  biburl    = {https://dblp.org/rec/journals/corr/LiTBZ15.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{Bahdanau2015,
archivePrefix = {arXiv},
arxivId = {1409.0473},
author = {Bahdanau, Dzmitry and Cho, Kyunghyun and Bengio, Yoshua},
booktitle = {ICLR},
doi = {10.1146/annurev.neuro.26.041002.131047},
eprint = {1409.0473},
isbn = {0147-006X (Print)},
issn = {0147-006X},
keywords = {Neural machine translation is a recently proposed,Unlike the traditional statistical machine transla,a source sentence into a fixed-length vector from,and propose to extend this by allowing a model to,bottleneck in improving the performance of this ba,for parts of a source sentence that are relevant t,having to form these parts as a hard segment expli,machine translation often belong to a family of en,maximize the translation performance. The models p,phrase-based system on the task of English-to-Fren,qualitative analysis reveals that the (soft-)align,the neural machine,translation aims at building a single neural netwo,translation. In this paper,we achieve a translation performance comparable to,we conjecture that the use of a fixed-length vecto,well with our intuition,without},
pages = {1--15},
pmid = {14527267},
title = {{Neural Machine Translation By Jointly Learning To Align and Translate}},
url = {http://arxiv.org/abs/1409.0473 http://arxiv.org/abs/1409.0473v3},
year = {2014}
}

@inproceedings{sutskever14sequence,
abstract = {Deep Neural Networks (DNNs) are powerful models that have achieved excellent performance on difficult learning tasks. Although DNNs work well whenever large labeled training sets are available, they cannot be used to map sequences to sequences. In this paper, we present a general end-to-end approach to sequence learning that makes minimal assumptions on the sequence structure. Our method uses a multilayered Long Short-Term Memory (LSTM) to map the input sequence to a vector of a fixed dimensionality, and then another deep LSTM to decode the target sequence from the vector. Our main result is that on an English to French translation task from the WMT'14 dataset, the translations produced by the LSTM achieve a BLEU score of 34.8 on the entire test set, where the LSTM's BLEU score was penalized on out-of-vocabulary words. Additionally, the LSTM did not have difficulty on long sentences. For comparison, a phrase-based SMT system achieves a BLEU score of 33.3 on the same dataset. When we used the LSTM to rerank the 1000 hypotheses produced by the aforementioned SMT system, its BLEU score increases to 36.5, which is close to the previous best result on this task. The LSTM also learned sensible phrase and sentence representations that are sensitive to word order and are relatively invariant to the active and the passive voice. Finally, we found that reversing the order of the words in all source sentences (but not target sentences) improved the LSTM's performance markedly, because doing so introduced many short term dependencies between the source and the target sentence which made the optimization problem easier.},
archivePrefix = {arXiv},
arxivId = {1409.3215},
author = {Sutskever, Ilya and Vinyals, Oriol and Le, Quoc V.},
booktitle = {NIPS},
eprint = {1409.3215},
isbn = {1409.3215},
pages = {9},
pmid = {2079951},
title = {{Sequence to Sequence Learning with Neural Networks}},
url = {http://arxiv.org/abs/1409.3215},
year = {2014}
}

@article{Xu2015,
abstract = {Inspired by recent work in machine translation and object detection, we introduce an attention based model that automatically learns to describe the content of images. We describe how we can train this model in a deterministic manner using standard backpropagation techniques and stochastically by maximizing a variational lower bound. We also show through visualization how the model is able to automatically learn to fix its gaze on salient objects while generating the corresponding words in the output sequence. We validate the use of attention with state-of-the-art performance on three benchmark datasets: Flickr8k, Flickr30k and MS COCO.},
archivePrefix = {arXiv},
arxivId = {1502.03044},
author = {Xu, Kelvin and Ba, Jimmy and Kiros, Ryan and Cho, Kyunghyun and Courville, Aaron and Salakhutdinov, Ruslan and Zemel, Richard and Bengio, Yoshua},
eprint = {1502.03044},
file = {:home/srush/.local/share/data/Mendeley Ltd./Mendeley Desktop/Downloaded/Xu et al. - 2015 - Show, Attend and Tell Neural Image Caption Generation with Visual Attention(2).pdf:pdf},
journal = {ICML},
month = {feb},
title = {{Show, Attend and Tell: Neural Image Caption Generation with Visual Attention}},
url = {http://arxiv.org/abs/1502.03044},
year = {2015}
}
@article{systran,
  title={SYSTRAN's Pure Neural Machine Translation System},
    author={Josep Crego and Jungi Kim and Jean Senellart},
      journal={arXiv preprint arXiv:1602.06023},
        year={2016}
        }
@InProceedings{Cho2014,
  title     = {{L}earning {P}hrase {R}epresentations using {RNN} {E}ncoder-{D}ecoder for {S}tatistical {M}achine {T}ranslation},
  author    = {Kyunghyun Cho and Bart van Merrienboer and Caglar Gulcehre and Dzmitry Bahdanau and Fethi Bougares and Holger Schwenk and Yoshua Bengio},
  booktitle = {Proc of EMNLP},
  year      = {2014}
}

@InProceedings{Luong2015,
  title     = {{E}ffective {A}pproaches to {A}ttention-based {N}eural {M}achine {T}ranslation},
  author    = {Minh-Thang Luong and Hieu Pham and Christopher D. Manning},
  booktitle = {Proc of EMNLP},
  year      = {2015}
}

@InProceedings{Luong2015b,
  title     = {{A}ddressing the {R}are {W}ord {P}roblem in {N}eural {M}achine {T}ranslation},
  author    = {Minh-Thang Luong and Ilya Sutskever and Quoc Le and Oriol Vinyals and Wojciech Zaremba},
  booktitle = {Proc of ACL},
  year      = {2015}
}

@article{wu2016google,
  title={Google's Neural Machine Translation System: Bridging the Gap between Human and Machine Translation},
    author={Wu, Yonghui and Schuster, Mike and Chen, Zhifeng and Le, Quoc V and Norouzi, Mohammad and Macherey, Wolfgang and Krikun, Maxim and Cao, Yuan and Gao, Qin and Macherey, Klaus and others},
      journal={arXiv preprint arXiv:1609.08144},
        year={2016}
        }


@inproceedings{dean2012large,
  title={Large scale distributed deep networks},
  author={Dean, Jeffrey and Corrado, Greg and Monga, Rajat and Chen, Kai and Devin, Matthieu and Mao, Mark and Senior, Andrew and Tucker, Paul and Yang, Ke and Le, Quoc V and others},
  booktitle={Advances in neural information processing systems},
  pages={1223--1231},
  year={2012}
}
@inproceedings{koehn2007moses,
  title={Moses: Open source toolkit for statistical machine translation},
    author={Koehn, Philipp and Hoang, Hieu and Birch, Alexandra and Callison-Burch, Chris and Federico, Marcello and Bertoldi, Nicola and Cowan, Brooke and Shen, Wade and Moran, Christine and Zens, Richard and others},
      booktitle={Proc ACL},
        pages={177--180},
          year={2007},
            organization={Association for Computational Linguistics}
            }

@inproceedings{dyer2010cdec,
  title={cdec: A decoder, alignment, and learning framework for finite-state and context-free translation models},
    author={Dyer, Chris and Weese, Jonathan and Setiawan, Hendra and Lopez, Adam and Ture, Ferhan and Eidelman, Vladimir and Ganitkevitch, Juri and Blunsom, Phil and Resnik, Philip},
      booktitle={Proc  ACL},
        pages={7--12},
          year={2010},
            organization={Association for Computational Linguistics}
            }

@article{hochreiter1997long,
  title={Long short-term memory},
  author={Hochreiter, Sepp and Schmidhuber, J{\"u}rgen},
  journal={Neural computation},
  volume={9},
  number={8},
  pages={1735--1780},
  year={1997},
  publisher={MIT Press}
}


@article{chung2014empirical,
  title={Empirical evaluation of gated recurrent neural networks on sequence modeling},
    author={Chung, Junyoung and Gulcehre, Caglar and Cho, KyungHyun and Bengio, Yoshua},
      journal={arXiv preprint arXiv:1412.3555},
        year={2014}
        }


@inproceedings{yang2016hierarchical,
  title={Hierarchical attention networks for document classification},
    author={Yang, Zichao and Yang, Diyi and Dyer, Chris and He, Xiaodong and Smola, Alex and Hovy, Eduard},
      booktitle={Proc ACL},
        year={2016}
        }

@article{martins2016softmax,
  title={From Softmax to Sparsemax: A Sparse Model of Attention and Multi-Label Classification},
    author={Martins, Andr{\'e} FT and Astudillo, Ram{\'o}n Fernandez},
      journal={arXiv preprint arXiv:1602.02068},
        year={2016}
        }

@article{DBLP:journals/corr/LeonardWW15,
  author    = {Nicholas L{\'{e}}onard and
                 Sagar Waghmare and
                                Yang Wang and
                                               Jin{-}Hwa Kim},
                                                 title     = {rnn : Recurrent Library for Torch},
                                                   journal   = {CoRR},
                                                     volume    = {abs/1511.07889},
                                                       year      = {2015},
                                                         url       = {http://arxiv.org/abs/1511.07889},
                                                           timestamp = {Wed, 23 Dec 2015 08:46:28 +0100},
                                                             biburl    = {http://dblp.uni-trier.de/rec/bib/journals/corr/LeonardWW15},
                                                               bibsource = {dblp computer science bibliography, http://dblp.org}
                                                               }


@inproceedings{DBLP:conf/conll/BowmanVVDJB16,
  author    = {Samuel R. Bowman and
                 Luke Vilnis and
                                Oriol Vinyals and
                                               Andrew M. Dai and
                                                              Rafal J{\'{o}}zefowicz and
                                                                             Samy Bengio},
                                                                               title     = {Generating Sentences from a Continuous Space},
                                                                                 booktitle = {Proceedings of the 20th {SIGNLL} Conference on Computational Natural
                                                                                                Language Learning, CoNLL 2016, Berlin, Germany, August 11-12, 2016},
                                                                                                  pages     = {10--21},
                                                                                                    year      = {2016},
                                                                                                      crossref  = {DBLP:conf/conll/2016},
                                                                                                        url       = {http://aclweb.org/anthology/K/K16/K16-1002.pdf},
                                                                                                          timestamp = {Sun, 04 Sep 2016 10:01:12 +0200},
                                                                                                            biburl    = {http://dblp.uni-trier.de/rec/bib/conf/conll/BowmanVVDJB16},
                                                                                                              bibsource = {dblp computer science bibliography, http://dblp.org}
                                                                                                              }

@inproceedings{DBLP:conf/nips/VinyalsBLKW16,
  author    = {Oriol Vinyals and
                 Charles Blundell and
                                Tim Lillicrap and
                                               Koray Kavukcuoglu and
                                                              Daan Wierstra},
                                                                title     = {Matching Networks for One Shot Learning},
                                                                  booktitle = {Advances in Neural Information Processing Systems 29: Annual Conference
                                                                                 on Neural Information Processing Systems 2016, December 5-10, 2016,
                                                                                                Barcelona, Spain},
                                                                                                  pages     = {3630--3638},
                                                                                                    year      = {2016},
                                                                                                      crossref  = {DBLP:conf/nips/2016},
                                                                                                        url       = {http://papers.nips.cc/paper/6385-matching-networks-for-one-shot-learning},
                                                                                                          timestamp = {Fri, 16 Dec 2016 19:45:58 +0100},
                                                                                                            biburl    = {http://dblp.uni-trier.de/rec/bib/conf/nips/VinyalsBLKW16},
                                                                                                              bibsource = {dblp computer science bibliography, http://dblp.org}
                                                                                                              }


@article{DBLP:journals/corr/WestonCB14,
  author    = {Jason Weston and
                 Sumit Chopra and
                                Antoine Bordes},
                                  title     = {Memory Networks},
                                    journal   = {CoRR},
                                      volume    = {abs/1410.3916},
                                        year      = {2014},
                                          url       = {http://arxiv.org/abs/1410.3916},
                                            timestamp = {Sun, 02 Nov 2014 11:25:59 +0100},
                                              biburl    = {http://dblp.uni-trier.de/rec/bib/journals/corr/WestonCB14},
                                                bibsource = {dblp computer science bibliography, http://dblp.org}
                                                }

@article{DBLP:journals/corr/XuBKCCSZB15,
  author    = {Kelvin Xu and
                 Jimmy Ba and
                                Ryan Kiros and
                                               Kyunghyun Cho and
                                                              Aaron C. Courville and
                                                                             Ruslan Salakhutdinov and
                                                                                            Richard S. Zemel and
                                                                                                           Yoshua Bengio},
                                                                                                             title     = {Show, Attend and Tell: Neural Image Caption Generation with Visual
                                                                                                                            Attention},
                                                                                                                              journal   = {CoRR},
                                                                                                                                volume    = {abs/1502.03044},
                                                                                                                                  year      = {2015},
                                                                                                                                    url       = {http://arxiv.org/abs/1502.03044},
                                                                                                                                      timestamp = {Mon, 02 Mar 2015 14:17:34 +0100},
                                                                                                                                        biburl    = {http://dblp.uni-trier.de/rec/bib/journals/corr/XuBKCCSZB15},
                                                                                                                                          bibsource = {dblp computer science bibliography, http://dblp.org}
                                                                                                                                          }



@article{DBLP:journals/corr/DengKR16,
  author    = {Yuntian Deng and
                 Anssi Kanervisto and
                                Alexander M. Rush},
                                  title     = {What You Get Is What You See: {A} Visual Markup Decompiler},
                                    journal   = {CoRR},
                                      volume    = {abs/1609.04938},
                                        year      = {2016},
                                          url       = {http://arxiv.org/abs/1609.04938},
                                            timestamp = {Mon, 03 Oct 2016 17:51:10 +0200},
                                              biburl    = {http://dblp.uni-trier.de/rec/bib/journals/corr/DengKR16},
                                                bibsource = {dblp computer science bibliography, http://dblp.org}
                                                }


@article{DBLP:journals/corr/ChanJLV15,
  author    = {William Chan and
                 Navdeep Jaitly and
                                Quoc V. Le and
                                               Oriol Vinyals},
                                                 title     = {Listen, Attend and Spell},
                                                   journal   = {CoRR},
                                                     volume    = {abs/1508.01211},
                                                       year      = {2015},
                                                         url       = {http://arxiv.org/abs/1508.01211},
                                                           timestamp = {Tue, 01 Sep 2015 14:42:40 +0200},
                                                             biburl    = {http://dblp.uni-trier.de/rec/bib/journals/corr/ChanJLV15},
                                                               bibsource = {dblp computer science bibliography, http://dblp.org}
                                                               }

@article{DBLP:journals/corr/SennrichHB15,
  author    = {Rico Sennrich and
                 Barry Haddow and
                                Alexandra Birch},
                                  title     = {Neural Machine Translation of Rare Words with Subword Units},
                                    journal   = {CoRR},
                                      volume    = {abs/1508.07909},
                                        year      = {2015},
                                          url       = {http://arxiv.org/abs/1508.07909},
                                            timestamp = {Tue, 01 Sep 2015 14:42:40 +0200},
                                              biburl    = {http://dblp.uni-trier.de/rec/bib/journals/corr/SennrichHB15},
                                                bibsource = {dblp computer science bibliography, http://dblp.org}
                                                }


@article{chopra2016abstractive,
  title={Abstractive sentence summarization with attentive recurrent neural networks},
    author={Chopra, Sumit and Auli, Michael and Rush, Alexander M and Harvard, SEAS},
      journal={Proceedings of NAACL-HLT16},
        pages={93--98},
          year={2016}
          }

@article{vinyals2015neural,
  title={A neural conversational model},
    author={Vinyals, Oriol and Le, Quoc},
      journal={arXiv preprint arXiv:1506.05869},
        year={2015}
        }

@inproceedings{neubig13travatar,
  title = {Travatar: A Forest-to-String Machine Translation Engine based on Tree Transducers},
    author = {Graham Neubig},
      booktitle = {Proc ACL },
        address = {Sofia, Bulgaria},
          month = {August},
            year = {2013}
            }

@ARTICLE{2017arXiv170301619N,
   author = {{Neubig}, G.},
       title = "{Neural Machine Translation and Sequence-to-sequence Models: A Tutorial}",
         journal = {ArXiv e-prints},
         archivePrefix = "arXiv",
            eprint = {1703.01619},
             primaryClass = "cs.CL",
              keywords = {Computer Science - Computation and Language, Computer Science - Learning, Statistics - Machine Learning},
                   year = 2017,
                       month = mar,
                          adsurl = {http://adsabs.harvard.edu/abs/2017arXiv170301619N},
                            adsnote = {Provided by the SAO/NASA Astrophysics Data System}
                            }

@article{DBLP:journals/corr/VaswaniSPUJGKP17,
  author    = {Ashish Vaswani and
               Noam Shazeer and
               Niki Parmar and
               Jakob Uszkoreit and
               Llion Jones and
               Aidan N. Gomez and
               Lukasz Kaiser and
               Illia Polosukhin},
  title     = {Attention Is All You Need},
  journal   = {CoRR},
  volume    = {abs/1706.03762},
  year      = {2017},
  url       = {http://arxiv.org/abs/1706.03762},
  archivePrefix = {arXiv},
  eprint    = {1706.03762},
  timestamp = {Mon, 13 Aug 2018 16:48:37 +0200},
  biburl    = {https://dblp.org/rec/bib/journals/corr/VaswaniSPUJGKP17},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@article{DBLP:journals/corr/GehringAGYD17,
  author    = {Jonas Gehring and
               Michael Auli and
               David Grangier and
               Denis Yarats and
               Yann N. Dauphin},
  title     = {Convolutional Sequence to Sequence Learning},
  journal   = {CoRR},
  volume    = {abs/1705.03122},
  year      = {2017},
  url       = {http://arxiv.org/abs/1705.03122},
  archivePrefix = {arXiv},
  eprint    = {1705.03122},
  timestamp = {Mon, 13 Aug 2018 16:48:03 +0200},
  biburl    = {https://dblp.org/rec/bib/journals/corr/GehringAGYD17},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@article{DBLP:journals/corr/abs-1709-02755,
  author    = {Tao Lei and
               Yu Zhang and
               Yoav Artzi},
  title     = {Training RNNs as Fast as CNNs},
  journal   = {CoRR},
  volume    = {abs/1709.02755},
  year      = {2017},
  url       = {http://arxiv.org/abs/1709.02755},
  archivePrefix = {arXiv},
  eprint    = {1709.02755},
  timestamp = {Mon, 13 Aug 2018 16:46:29 +0200},
  biburl    = {https://dblp.org/rec/bib/journals/corr/abs-1709-02755},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@article{DBLP:journals/corr/SeeLM17,
  author    = {Abigail See and
               Peter J. Liu and
               Christopher D. Manning},
  title     = {Get To The Point: Summarization with Pointer-Generator Networks},
  journal   = {CoRR},
  volume    = {abs/1704.04368},
  year      = {2017},
  url       = {http://arxiv.org/abs/1704.04368},
  archivePrefix = {arXiv},
  eprint    = {1704.04368},
  timestamp = {Mon, 13 Aug 2018 16:46:08 +0200},
  biburl    = {https://dblp.org/rec/bib/journals/corr/SeeLM17},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@article{DBLP:journals/corr/abs-1805-00631,
  author    = {Biao Zhang and
               Deyi Xiong and
               Jinsong Su},
  title     = {Accelerating Neural Transformer via an Average Attention Network},
  journal   = {CoRR},
  volume    = {abs/1805.00631},
  year      = {2018},
  url       = {http://arxiv.org/abs/1805.00631},
  archivePrefix = {arXiv},
  eprint    = {1805.00631},
  timestamp = {Mon, 13 Aug 2018 16:46:01 +0200},
  biburl    = {https://dblp.org/rec/bib/journals/corr/abs-1805-00631},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@article{DBLP:journals/corr/MartinsA16,
  author    = {Andr{\'{e}} F. T. Martins and
               Ram{\'{o}}n Fern{\'{a}}ndez Astudillo},
  title     = {From Softmax to Sparsemax: {A} Sparse Model of Attention and Multi-Label
               Classification},
  journal   = {CoRR},
  volume    = {abs/1602.02068},
  year      = {2016},
  url       = {http://arxiv.org/abs/1602.02068},
  archivePrefix = {arXiv},
  eprint    = {1602.02068},
  timestamp = {Mon, 13 Aug 2018 16:49:13 +0200},
  biburl    = {https://dblp.org/rec/bib/journals/corr/MartinsA16},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{garg2019jointly,
  title = {Jointly Learning to Align and Translate with Transformer Models},
  author = {Garg, Sarthak and Peitz, Stephan and Nallasamy, Udhyakumar and Paulik, Matthias},
  booktitle = {Conference on Empirical Methods in Natural Language Processing (EMNLP)},
  address = {Hong Kong},
  month = {November},
  url = {https://arxiv.org/abs/1909.02074},
  year = {2019},
}

@inproceedings{DeeperTransformer,
    title = "Learning Deep Transformer Models for Machine Translation",
    author = "Wang, Qiang  and
      Li, Bei  and
      Xiao, Tong  and
      Zhu, Jingbo  and
      Li, Changliang  and
      Wong, Derek F.  and
      Chao, Lidia S.",
    booktitle = "Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics",
    month = jul,
    year = "2019",
    address = "Florence, Italy",
    publisher = "Association for Computational Linguistics",
    url = "https://www.aclweb.org/anthology/P19-1176",
    doi = "10.18653/v1/P19-1176",
    pages = "1810--1822",
    abstract = "Transformer is the state-of-the-art model in recent machine translation evaluations. Two strands of research are promising to improve models of this kind: the first uses wide networks (a.k.a. Transformer-Big) and has been the de facto standard for development of the Transformer system, and the other uses deeper language representation but faces the difficulty arising from learning deep networks. Here, we continue the line of research on the latter. We claim that a truly deep Transformer model can surpass the Transformer-Big counterpart by 1) proper use of layer normalization and 2) a novel way of passing the combination of previous layers to the next. On WMT{'}16 English-German and NIST OpenMT{'}12 Chinese-English tasks, our deep system (30/25-layer encoder) outperforms the shallow Transformer-Big/Base baseline (6-layer encoder) by 0.4-2.4 BLEU points. As another bonus, the deep model is 1.6X smaller in size and 3X faster in training than Transformer-Big.",
}
